#include "tokenizer.hpp"
#include <iostream>

int main() {
    std::unique_ptr<Tokenizer> tokenizer = create_tokenizer("resources/hamlet.model");
    std::string text = "To be, or not to be: that is the question";
    std::cout << "Original text: " << text << std::endl;

    std::vector<int> tokens = tokenizer->encode(text);
    std::cout << "Encoded tokens: ";
    for (int token : tokens) {
        std::cout << token << " ";
    }
    std::cout << std::endl;

    std::string decoded_text = tokenizer->decode(tokens);
    std::cout << "Decoded text (should be the same as the original): " << decoded_text << std::endl;

    return 0;
}
